import os

# 用于分割多种文档的工具类

class DocSplit:
    def __init__(self):
        pass


    # txt文档风格按行分割
    def Text_split(self, path):
        result = []
        with open(path, 'r', encoding='utf-8') as f:
            for line in f.readlines():
                if line != "\n":
                    result.append(line.strip("\n"))

        return result






if __name__ == '__main__':
    from tools.Pgvector_op import Pgvector
    from tools.Transformer_sentence import BegZh
    import configparser
    ds = DocSplit()
    files=ds.Text_split('../src/test/uploads/奖学金问题.txt')


    cf = configparser.ConfigParser()
    cf.read("../config/config.ini")
    beg_zh = BegZh(cf.get("beg_model", "path"))


    pg = Pgvector(
        cf.get("RAG", "host"),
        cf.get("RAG", "port"),
        cf.get("RAG", "database"),
        cf.get("RAG", "user"),
        cf.get("RAG", "password"),
        cf.get("RAG", "table"),
    )


    data = beg_zh.encode_docs(files,"奖学金问题.txt")

    pg.insert(data)




    # loader = TextLoader("../src/test/uploads/奖学金问题.txt", encoding="utf-8")
    # documents = loader.load()
    # text_splitter = CharacterTextSplitter(
    #     separator="\n\n",
    #     chunk_size=75,
    #     chunk_overlap=30
    # )
    # docs = text_splitter.split_documents(documents)
    # print(docs)